{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.70517814,  0.70903015, -0.1703155 ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkbklEQVR4nO3dfXRU9YH/8c+dTDI8hJnwYGaIJEKrBSmCyuPUc9ptyZp2syJCz3Ety6Yuq4KBirield2Kpz09Gw7ur61SQa1nhbY87GFXUFiom1/AuC3hKRDlyfiEJBInUWlmkkAmD/P9/UGZn4OgSWCSb8L7dc6cY+79ztzv3JJ59965mXGMMUYAAFjI1dMTAADgUogUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaPRapZ555RiNHjlS/fv00depU7du3r6emAgCwVI9E6j/+4z+0ZMkSPfHEEzp48KAmTJigvLw81dXV9cR0AACWcnriA2anTp2qyZMn61e/+pUkKRaLKTs7W4sWLdJjjz3W3dMBAFjK3d0bbGlpUXl5uZYuXRpf5nK5lJubq7KysoveJxqNKhqNxn+OxWI6ffq0hg4dKsdxkj5nAMCVZYxRQ0ODsrKy5HJd+qRet0fqk08+UXt7u/x+f8Jyv9+vt95666L3KSoq0k9+8pPumB4AoBtVV1drxIgRl1zf7ZHqiqVLl2rJkiXxn8PhsHJyclRdXS2v19uDMwMAdEUkElF2drYGDRr0heO6PVLDhg1TSkqKamtrE5bX1tYqEAhc9D4ej0cej+dzy71eL5ECgF7sy96y6far+9LS0jRx4kSVlJTEl8ViMZWUlCgYDHb3dAAAFuuR031LlixRQUGBJk2apClTpuiXv/ylmpqadO+99/bEdAAAluqRSN199936+OOPtWzZMoVCId188836/e9//7mLKQAAV7ce+TupyxWJROTz+RQOh3lPCgB6oY6+jvPZfQAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCs1elIvf7667rjjjuUlZUlx3G0ZcuWhPXGGC1btkzDhw9X//79lZubq3feeSdhzOnTpzVnzhx5vV5lZGRo3rx5amxsvKwnAgDoezodqaamJk2YMEHPPPPMRdevWLFCTz/9tJ599lnt3btXAwcOVF5enpqbm+Nj5syZo6NHj6q4uFjbtm3T66+/rvvvv7/rzwIA0DeZyyDJbN68Of5zLBYzgUDAPPnkk/Fl9fX1xuPxmA0bNhhjjDl27JiRZPbv3x8fs2PHDuM4jjl16lSHthsOh40kEw6HL2f6AIAe0tHX8Sv6ntSJEycUCoWUm5sbX+bz+TR16lSVlZVJksrKypSRkaFJkybFx+Tm5srlcmnv3r0XfdxoNKpIJJJwAwD0fVc0UqFQSJLk9/sTlvv9/vi6UCikzMzMhPVut1tDhgyJj7lQUVGRfD5f/JadnX0lpw0AsFSvuLpv6dKlCofD8Vt1dXVPTwkA0A2uaKQCgYAkqba2NmF5bW1tfF0gEFBdXV3C+ra2Np0+fTo+5kIej0derzfhBgDo+65opEaNGqVAIKCSkpL4skgkor179yoYDEqSgsGg6uvrVV5eHh+zc+dOxWIxTZ069UpOBwDQy7k7e4fGxka9++678Z9PnDihiooKDRkyRDk5OVq8eLF+9rOf6YYbbtCoUaP0+OOPKysrSzNnzpQk3Xjjjfrud7+r++67T88++6xaW1u1cOFC/c3f/I2ysrKu2BMDAPQBnb1scNeuXUbS524FBQXGmHOXoT/++OPG7/cbj8djpk+fbiorKxMe49NPPzX33HOPSU9PN16v19x7772moaHhil+6CACwU0dfxx1jjOnBRnZJJBKRz+dTOBzm/SkA6IU6+jreK67uAwBcnYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBa7p6eAHApxhiZtjbFWlpk2tulWEyOyyXH7ZYrLU2Om3++QF/HbzmsFGtp0dmTJ1W/Z48iFRVqrqlR7OxZpQwcqP7Z2fJNmSLfrbeqX3a2nJSUnp4ugCQhUrBOLBpV3fbt+qS4WNFTpyRj4uvaGxrUeOyYGisrdfr115V5xx0a+u1vy3Fx5hroi/jNhlVi0ahqNmxQzbp1in74YUKgErS36+z776v6uef0/ooVamtqkrnUWAC9FkdSsEaspUV127erbts2mZaWjt2nuVn1u3fLcRxlz58vt9crx3GSPFMA3YUjKVjBGKOmykrVvfxyhwP1WX/64x9VtWqVTFsbR1RAH0KkYAXT2qo/7dmj1tOnu/wY9Xv26MSTT6otHCZUQB/B6T5Yof3MGX28devlPYgxqt+zR0bSdQsWyJ2Rwak/oJfjSAp9TnjPHp185hnFmps5ogJ6OSKFPim8b59O/Nu/qeWTTwgV0IsRKfRZ4f37Vb16tVr/9KeengqALiJSsEcS/iA3fOCATv7qV2qNRDiiAnohIgUruNPTNfyee5Ly2JEDB/TBz3+uaChEqIBehkjBCo7bLd8tt8hz7bVJefzIwYOqfu45tXz6aVIeH0ByEClYY8BXv6rA978vV79+SXn8yMGDqlq5kospgF6ESMEaTkqKhn772/LeemvSthE5dEgfPPWUmj/8kFABvQCRglUcl0vXLVqkjG98I2nbaHjjjXOn/urqkrYNAFcGkYJ1UgYMUM6CBcoIBqUkfWJEw5tv6uTKlWr+6COOqACLESlYx3Ecub1ejfrHf1TG1KlJ2875UJ394ANCBViKSMFKjuPIcbuV8+CD8k2blrTtNB45ournn1c0FEraNgB0HZGCtRzHkdvn03ULFsg3ZUrSttN49KhOPv20zp48KROLJW07ADqPSMFqjuPInZGhUY88It/kyUnbTuPRozq5apXOnDjBqT/AIkQK1nMcR65+/ZQ9f758kyYlbTtNx4+fO/V36hShAixBpNArOI6jtGHDlFNYKO/EiUnbTtPx4/pg5Uqdee89Tv0BFiBS6DUcx1Ha0KEauXixvLfckrTtNB0/rurnntOZd9/liAroYUQKvY7b61X2Aw8kN1SVlar+9a91tqqKUAE9iEih13EcR57hw5WzcKEG3Xxz0rbTVFmpkytXqqmyklN/QA8hUuiVHMeR55prNPJHP9KgCROStp0zb7+t6ueeU9Pbb3NEBfQAIoVeLXXoUGXfd58GjR+ftG2cee89Vf/61zrz/vuECuhmnYpUUVGRJk+erEGDBikzM1MzZ85UZWVlwpjm5mYVFhZq6NChSk9P1+zZs1VbW5swpqqqSvn5+RowYIAyMzP16KOPqq2t7fKfDa46juOoX3a2rlu0KLmheucdnVy5Uo1HjnDqD+hGnYpUaWmpCgsLtWfPHhUXF6u1tVW33367mpqa4mMefvhhbd26VZs2bVJpaalqamo0a9as+Pr29nbl5+erpaVFu3fv1tq1a7VmzRotW7bsyj0rXFUcx5HH71dOYaHSx41L2nbOvv++ql94QY1vvZW0bQBI5JjLOH/x8ccfKzMzU6WlpfrmN7+pcDisa665RuvXr9f3v/99SdJbb72lG2+8UWVlZZo2bZp27Nihv/7rv1ZNTY38fr8k6dlnn9U//dM/6eOPP1ZaWtqXbjcSicjn8ykcDsvr9XZ1+uhjjDE6+8EHqn7+eTUePZq07fT/yleU88ADGjhmjJwkfUo70Nd19HX8st6TCofDkqQhQ4ZIksrLy9Xa2qrc3Nz4mDFjxignJ0dlZWWSpLKyMt10003xQElSXl6eIpGIjl7ihSUajSoSiSTcgAs5jqP+I0fquh/9SOlf/3rStnP2/fd1ctUqNVRUcOoPSLIuRyoWi2nx4sW67bbbNO7Pp1hCoZDS0tKUkZGRMNbv9yv050+ZDoVCCYE6v/78uospKiqSz+eL37Kzs7s6bfRxjuOo3/Dhypk/XwNvvDFp22k+eVLVL7ygum3bkrYNAJcRqcLCQh05ckQbN268kvO5qKVLlyocDsdv1dXVSd8merd+2dnKvv/+5IaqulqnfvMbhTZvlmlvT9p2gKtZlyK1cOFCbdu2Tbt27dKIESPiywOBgFpaWlRfX58wvra2VoFAID7mwqv9zv98fsyFPB6PvF5vwg34Io7LpQFf+YpGLlqkgWPGJG07pqVFp9asUe3mzZz6A5KgU5EyxmjhwoXavHmzdu7cqVGjRiWsnzhxolJTU1VSUhJfVllZqaqqKgWDQUlSMBjU4cOHVVdXFx9TXFwsr9ersWPHXs5zARI4jiPPtdeeO6L62teStyFjdOp3v1PdK68kbxvAVapTV/c9+OCDWr9+vV5++WWNHj06vtzn86l///6SpAULFmj79u1as2aNvF6vFi1aJEnavXu3pHOXoN98883KysrSihUrFAqFNHfuXP3DP/yD/vVf/7VD8+DqPnSGicV05t13Vf3882p6++2kbcdJS1PWD34g/4wZctzupG0H6As6+jreqUhd6nLbF198UT/84Q8lnftj3kceeUQbNmxQNBpVXl6eVq1alXAq7+TJk1qwYIFee+01DRw4UAUFBVq+fLncHfzFJlLoLGOMzlZV6eTKlTqTxFBJUtbcuQrMni3HxQe6AJeSlEjZgkihK4wxanrrLVU//7zOvPde8jaUkqIRBQXyz5yZvG0AvVy3/J0U0Js4jqOBo0cr+4EHNOD665O3ofZ2nfrtbxX6r/9SjI/7Ai4LkcJVxXG5NHD0aOU8+KD6f+UrSduOaW3VqbVrVffyy3woLXAZiBSuOo7jaMBXv6rsefPUf+TIpG6rZt061b3yCqECuohI4arkOI7Sv/51ZT/wQHKPqNradOo3v1HtSy9x6g/oAiKFq5bjcmnQ17+unAceUL/rrkvadjj1B3QdkcJVb+CYMQp85utkkqVm/XpO/QGdRKRw1XMcR0O++U1de++9cjrwVTFdZVpbz32EElf9AR1GpABJTkqKAnfdpaw5c6QkfkeUaW/Xqd/8ho9QAjqISAGf4Z8xQ1l/+7dJDZV07tRf7ZYtfCgt8CWIFPAZTkqKArNm6dof/jC5p/5aWvThmjWqfeklGU79AZdEpIALxE/9/eAHyd1QLHbu8vStW5O7HaAX46OagUvwz5gh096umvXrpSR+qWHNunVyJGXOmCEnJSVp2wF6I46kgEtw3G4FZs/WiIICOampSdvO+VN/fMMv8HlECvgCjssl/8yZyT/1Z4xqfvtb1XLVH5CA031AB2Teeee5kKxfn7wLHYxRzbp1kiT/HXfwxYmAOJICOsTldss/e7au7YZTf6defPHcqT8uTweIFNBRjuMoc8aM5J/6058/Pf3ll5O+HcB2nE8AOsFxnHOn/nTuD3JNa2tyNhSL6dS6dTLGKHPGDLk49YerFEdSQCe53G75Z83StXPnJvWScdPSolNr1qhuyxY+lBZXLSIFdMH5I6qsOXOSvq3zp/4IFa5GnEMAushxHGXOnCk5jmo2bJBpaUnKdkx7u0799rcysRin/nDV4UgKuAyuP//Bb9acOZIreb9O57/mgyMqXG2IFHAF+Lvr1B9fnIirDJECrgDH5VLgrruS/+npra36+NVX1Xj8OKHCVYFIAVeI43YrMGvWZX9x4qmmJm2rrtaG99/X/62pUdMFl7lHP/xQH23YoPbGxsudMmA93oEFrjD/jBkybW2q+d3vpE4c7RhjdKKxUU8cOqQPGhvV3N4ub2qqxg0erH+bPFmpn3nPq+GNN3Tmvfc0aMIEOUn+gkagJ3EkBVxhXf3ixPcbG3XfH/+o4+Gwzra3y0gKt7bqj3V1emjvXn3a3JwwvurZZ6/wzAH7ECkgCbryxYm/PHpU4Ut8gsW+Tz5RcU1NwrKkfdoFYBEiBSSRf8YMZc2dm9TL04G+jN8cIIkSvjgxiVf9AX0VkQKSzHG55O/Aqb/87GylXuIiiJHp6Ro/ZEgypgdYjav7gG6SOWOGFIupZt26i35NfF5WliTpZ2+8oZb2dsUkpTiOMtLS9H8mT9Z16ekJ4/2zZ3fHtIEeRaSAbnL+ixMdt/vcZ/FdcOGD4zjKy8rSiAEDtO3DD/Vpc7NGpqfr7lGjNNTjSRjrGT5cg4NBLj9Hn0ekgG50/tPTTXu7Tq1de9H14wYP1rjBgy/5GO7Bg5U1d67cXm8ypwpYgUgB3Sz+xYnGqGbDBrk8HsWam2Xa2r70vinp6Rp+993KmDIlqd9lBdiCSAE9IH7qLzVVg2+7TR+/+qpO79qllrq6i453UlLULztb/rvu0pC/+AtO8+Gq4Zhe+CmVkUhEPp9P4XBYXk55oA+IRaNqevtt/Wn3bjUePapoKKRYNKqU9HT1GzFCvkmT5Js0Sf1zcjiCQp/Q0ddxjqQAC7g8HqWPG6eBX/ua2s+elWlrk4nF5KSkyJWaKteAAXzZIa5K/KsHLOE4jhyPR64LruQDrmb8MS8AwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1OhWp1atXa/z48fJ6vfJ6vQoGg9qxY0d8fXNzswoLCzV06FClp6dr9uzZqq2tTXiMqqoq5efna8CAAcrMzNSjjz6qtra2K/NsAAB9SqciNWLECC1fvlzl5eU6cOCAvvOd7+jOO+/U0aNHJUkPP/ywtm7dqk2bNqm0tFQ1NTWaNWtW/P7t7e3Kz89XS0uLdu/erbVr12rNmjVatmzZlX1WAIC+wVymwYMHmxdeeMHU19eb1NRUs2nTpvi648ePG0mmrKzMGGPM9u3bjcvlMqFQKD5m9erVxuv1mmg02uFthsNhI8mEw+HLnT4AoAd09HW8y+9Jtbe3a+PGjWpqalIwGFR5eblaW1uVm5sbHzNmzBjl5OSorKxMklRWVqabbrpJfr8/PiYvL0+RSCR+NHYx0WhUkUgk4QYA6Ps6HanDhw8rPT1dHo9H8+fP1+bNmzV27FiFQiGlpaUpIyMjYbzf71coFJIkhUKhhECdX39+3aUUFRXJ5/PFb9nZ2Z2dNgCgF+p0pEaPHq2Kigrt3btXCxYsUEFBgY4dO5aMucUtXbpU4XA4fquurk7q9gAAdnB39g5paWm6/vrrJUkTJ07U/v379dRTT+nuu+9WS0uL6uvrE46mamtrFQgEJEmBQED79u1LeLzzV/+dH3MxHo9HHo+ns1MFAPRyl/13UrFYTNFoVBMnTlRqaqpKSkri6yorK1VVVaVgMChJCgaDOnz4sOrq6uJjiouL5fV6NXbs2MudCgCgj+nUkdTSpUv1ve99Tzk5OWpoaND69ev12muv6dVXX5XP59O8efO0ZMkSDRkyRF6vV4sWLVIwGNS0adMkSbfffrvGjh2ruXPnasWKFQqFQvrxj3+swsJCjpQAAJ/TqUjV1dXp7/7u7/TRRx/J5/Np/PjxevXVV/WXf/mXkqRf/OIXcrlcmj17tqLRqPLy8rRq1ar4/VNSUrRt2zYtWLBAwWBQAwcOVEFBgX76059e2WcFAOgTHGOM6elJdFYkEpHP51M4HJbX6+3p6QAAOqmjr+N8dh8AwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAa11WpJYvXy7HcbR48eL4submZhUWFmro0KFKT0/X7NmzVVtbm3C/qqoq5efna8CAAcrMzNSjjz6qtra2y5kKAKAP6nKk9u/fr+eee07jx49PWP7www9r69at2rRpk0pLS1VTU6NZs2bF17e3tys/P18tLS3avXu31q5dqzVr1mjZsmVdfxYAgL7JdEFDQ4O54YYbTHFxsfnWt75lHnroIWOMMfX19SY1NdVs2rQpPvb48eNGkikrKzPGGLN9+3bjcrlMKBSKj1m9erXxer0mGo12aPvhcNhIMuFwuCvTBwD0sI6+jnfpSKqwsFD5+fnKzc1NWF5eXq7W1taE5WPGjFFOTo7KysokSWVlZbrpppvk9/vjY/Ly8hSJRHT06NGLbi8ajSoSiSTcAAB9n7uzd9i4caMOHjyo/fv3f25dKBRSWlqaMjIyEpb7/X6FQqH4mM8G6vz68+supqioSD/5yU86O1UAQC/XqSOp6upqPfTQQ1q3bp369euXrDl9ztKlSxUOh+O36urqbts2AKDndCpS5eXlqqur06233iq32y23263S0lI9/fTTcrvd8vv9amlpUX19fcL9amtrFQgEJEmBQOBzV/ud//n8mAt5PB55vd6EGwCg7+tUpKZPn67Dhw+roqIifps0aZLmzJkT/+/U1FSVlJTE71NZWamqqioFg0FJUjAY1OHDh1VXVxcfU1xcLK/Xq7Fjx16hpwUA6As69Z7UoEGDNG7cuIRlAwcO1NChQ+PL582bpyVLlmjIkCHyer1atGiRgsGgpk2bJkm6/fbbNXbsWM2dO1crVqxQKBTSj3/8YxUWFsrj8VyhpwUA6As6feHEl/nFL34hl8ul2bNnKxqNKi8vT6tWrYqvT0lJ0bZt27RgwQIFg0ENHDhQBQUF+ulPf3qlpwIA6OUcY4zp6Ul0ViQSkc/nUzgc5v0pAOiFOvo6zmf3AQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCs5e7pCXSFMUaSFIlEengmAICuOP/6ff71/FJ6ZaQ+/fRTSVJ2dnYPzwQAcDkaGhrk8/kuub5XRmrIkCGSpKqqqi98cle7SCSi7OxsVVdXy+v19vR0rMV+6hj2U8ewnzrGGKOGhgZlZWV94bheGSmX69xbaT6fj38EHeD1etlPHcB+6hj2U8ewn75cRw4yuHACAGAtIgUAsFavjJTH49ETTzwhj8fT01OxGvupY9hPHcN+6hj205XlmC+7/g8AgB7SK4+kAABXByIFALAWkQIAWItIAQCs1Ssj9cwzz2jkyJHq16+fpk6dqn379vX0lLrV66+/rjvuuENZWVlyHEdbtmxJWG+M0bJlyzR8+HD1799fubm5eueddxLGnD59WnPmzJHX61VGRobmzZunxsbGbnwWyVVUVKTJkydr0KBByszM1MyZM1VZWZkwprm5WYWFhRo6dKjS09M1e/Zs1dbWJoypqqpSfn6+BgwYoMzMTD366KNqa2vrzqeSVKtXr9b48ePjf3gaDAa1Y8eO+Hr20cUtX75cjuNo8eLF8WXsqyQxvczGjRtNWlqa+fd//3dz9OhRc99995mMjAxTW1vb01PrNtu3bzf/8i//Yl566SUjyWzevDlh/fLly43P5zNbtmwxb7zxhpkxY4YZNWqUOXv2bHzMd7/7XTNhwgSzZ88e87//+7/m+uuvN/fcc083P5PkycvLMy+++KI5cuSIqaioMH/1V39lcnJyTGNjY3zM/PnzTXZ2tikpKTEHDhww06ZNM9/4xjfi69va2sy4ceNMbm6uOXTokNm+fbsZNmyYWbp0aU88paR45ZVXzH//93+bt99+21RWVpp//ud/NqmpqebIkSPGGPbRxezbt8+MHDnSjB8/3jz00EPx5eyr5Oh1kZoyZYopLCyM/9ze3m6ysrJMUVFRD86q51wYqVgsZgKBgHnyySfjy+rr643H4zEbNmwwxhhz7NgxI8ns378/PmbHjh3GcRxz6tSpbpt7d6qrqzOSTGlpqTHm3D5JTU01mzZtio85fvy4kWTKysqMMef+z4DL5TKhUCg+ZvXq1cbr9ZpoNNq9T6AbDR482Lzwwgvso4toaGgwN9xwgykuLjbf+ta34pFiXyVPrzrd19LSovLycuXm5saXuVwu5ebmqqysrAdnZo8TJ04oFAol7COfz6epU6fG91FZWZkyMjI0adKk+Jjc3Fy5XC7t3bu32+fcHcLhsKT//+HE5eXlam1tTdhPY8aMUU5OTsJ+uummm+T3++Nj8vLyFIlEdPTo0W6cffdob2/Xxo0b1dTUpGAwyD66iMLCQuXn5yfsE4l/T8nUqz5g9pNPPlF7e3vC/8iS5Pf79dZbb/XQrOwSCoUk6aL76Py6UCikzMzMhPVut1tDhgyJj+lLYrGYFi9erNtuu03jxo2TdG4fpKWlKSMjI2HshfvpYvvx/Lq+4vDhwwoGg2publZ6ero2b96ssWPHqqKign30GRs3btTBgwe1f//+z63j31Py9KpIAV1RWFioI0eO6A9/+ENPT8VKo0ePVkVFhcLhsP7zP/9TBQUFKi0t7elpWaW6uloPPfSQiouL1a9fv56ezlWlV53uGzZsmFJSUj53xUxtba0CgUAPzcou5/fDF+2jQCCgurq6hPVtbW06ffp0n9uPCxcu1LZt27Rr1y6NGDEivjwQCKilpUX19fUJ4y/cTxfbj+fX9RVpaWm6/vrrNXHiRBUVFWnChAl66qmn2EefUV5errq6Ot16661yu91yu90qLS3V008/LbfbLb/fz75Kkl4VqbS0NE2cOFElJSXxZbFYTCUlJQoGgz04M3uMGjVKgUAgYR9FIhHt3bs3vo+CwaDq6+tVXl4eH7Nz507FYjFNnTq12+ecDMYYLVy4UJs3b9bOnTs1atSohPUTJ05Uampqwn6qrKxUVVVVwn46fPhwQtCLi4vl9Xo1duzY7nkiPSAWiykajbKPPmP69Ok6fPiwKioq4rdJkyZpzpw58f9mXyVJT1+50VkbN240Ho/HrFmzxhw7dszcf//9JiMjI+GKmb6uoaHBHDp0yBw6dMhIMj//+c/NoUOHzMmTJ40x5y5Bz8jIMC+//LJ58803zZ133nnRS9BvueUWs3fvXvOHP/zB3HDDDX3qEvQFCxYYn89nXnvtNfPRRx/Fb2fOnImPmT9/vsnJyTE7d+40Bw4cMMFg0ASDwfj685cM33777aaiosL8/ve/N9dcc02fumT4scceM6WlpebEiRPmzTffNI899phxHMf8z//8jzGGffRFPnt1nzHsq2TpdZEyxpiVK1eanJwck5aWZqZMmWL27NnT01PqVrt27TKSPncrKCgwxpy7DP3xxx83fr/feDweM336dFNZWZnwGJ9++qm55557THp6uvF6vebee+81DQ0NPfBskuNi+0eSefHFF+Njzp49ax588EEzePBgM2DAAHPXXXeZjz76KOFxPvjgA/O9733P9O/f3wwbNsw88sgjprW1tZufTfL8/d//vbnuuutMWlqaueaaa8z06dPjgTKGffRFLowU+yo5+KoOAIC1etV7UgCAqwuRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1vp/D2n3UGgMno8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3569]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 1),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "\n",
    "    def forward(self, state):\n",
    "        return self.sequential(state) * 2.0\n",
    "\n",
    "\n",
    "model_action = Model()\n",
    "model_action_next = Model()\n",
    "\n",
    "model_action_next.load_state_dict(model_action.state_dict())\n",
    "\n",
    "model_action(torch.randn(1, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0948]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_value = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "model_value_next = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 1),\n",
    ")\n",
    "\n",
    "model_value_next.load_state_dict(model_value.state_dict())\n",
    "\n",
    "model_value(torch.randn(1, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3161582674136165"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 3)\n",
    "    action = model_action(state).item()\n",
    "    #给动作添加噪声,增加探索\n",
    "    action += random.normalvariate(mu=0, sigma=0.01)\n",
    "    return action\n",
    "\n",
    "\n",
    "get_action([1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,\n",
       " (array([ 0.99298847, -0.11821105, -0.09574469], dtype=float32),\n",
       "  0.44153784519818506,\n",
       "  -0.015151092194003398,\n",
       "  array([ 0.9922727 , -0.12407614, -0.1181723 ], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#样本池\n",
    "datas = []\n",
    "\n",
    "\n",
    "#向样本池中添加N条数据,删除M条最古老的数据\n",
    "def update_data():\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step([action])\n",
    "\n",
    "        #记录数据样本\n",
    "        datas.append((state, action, reward, next_state, over))\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    #数据上限,超出时从最古老的开始删除\n",
    "    while len(datas) > 10000:\n",
    "        datas.pop(0)\n",
    "\n",
    "\n",
    "update_data()\n",
    "\n",
    "len(datas), datas[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2036/1710091499.py:7: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.1469, -0.9891,  1.7092],\n",
       "         [ 0.9923, -0.1241, -0.1182],\n",
       "         [-0.9136,  0.4065,  5.6514],\n",
       "         [-0.9862, -0.1656, -5.9019],\n",
       "         [-0.0980, -0.9952, -3.6718]]),\n",
       " tensor([[0.4019],\n",
       "         [0.4501],\n",
       "         [0.5536],\n",
       "         [1.0190],\n",
       "         [0.8703]]),\n",
       " tensor([[ -2.3182],\n",
       "         [ -0.0171],\n",
       "         [-10.6085],\n",
       "         [-12.3365],\n",
       "         [ -4.1344]]),\n",
       " tensor([[ 0.1976, -0.9803,  1.0277],\n",
       "         [ 0.9914, -0.1312, -0.1437],\n",
       "         [-0.9932,  0.1164,  6.0393],\n",
       "         [-0.9919,  0.1270, -5.8732],\n",
       "         [-0.3075, -0.9516, -4.2876]]),\n",
       " tensor([[0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0],\n",
       "         [0]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取一批数据样本\n",
    "def get_sample():\n",
    "    #从样本池中采样\n",
    "    samples = random.sample(datas, 64)\n",
    "\n",
    "    #[b, 3]\n",
    "    state = torch.FloatTensor([i[0] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    action = torch.FloatTensor([i[1] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 1]\n",
    "    reward = torch.FloatTensor([i[2] for i in samples]).reshape(-1, 1)\n",
    "    #[b, 3]\n",
    "    next_state = torch.FloatTensor([i[3] for i in samples]).reshape(-1, 3)\n",
    "    #[b, 1]\n",
    "    over = torch.LongTensor([i[4] for i in samples]).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "state[:5], action[:5], reward[:5], next_state[:5], over[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1208.2554823965668"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step([action])\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play and random.random() < 0.2:  #跳帧\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.1190],\n",
       "         [ 0.0410],\n",
       "         [ 0.3272],\n",
       "         [-0.2255],\n",
       "         [-0.1502]], grad_fn=<SliceBackward0>),\n",
       " tensor([[ -2.2370],\n",
       "         [  0.0221],\n",
       "         [-10.2570],\n",
       "         [-12.5429],\n",
       "         [ -4.3034]], grad_fn=<SliceBackward0>))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_value(state, action):\n",
    "    #直接评估综合了state和action的value\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([state, action], dim=1)\n",
    "\n",
    "    #[b, 4] -> [b, 1]\n",
    "    return model_value(input)\n",
    "\n",
    "\n",
    "def get_target(next_state, reward, over):\n",
    "    #对next_state的评估需要先把它对应的动作计算出来,这里用model_action_next来计算\n",
    "    #[b, 3] -> [b, 1]\n",
    "    action = model_action_next(next_state)\n",
    "\n",
    "    #和value的计算一样,action拼合进next_state里综合计算\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([next_state, action], dim=1)\n",
    "\n",
    "    #[b, 4] -> [b, 1]\n",
    "    target = model_value_next(input) * 0.98\n",
    "\n",
    "    #[b, 1] * [b, 1] -> [b, 1]\n",
    "    target *= (1 - over)\n",
    "\n",
    "    #[b, 1] + [b, 1] -> [b, 1]\n",
    "    target += reward\n",
    "\n",
    "    return target\n",
    "\n",
    "\n",
    "get_value(state, action)[:5], get_target(next_state, reward, over)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.0231, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_loss_action(state):\n",
    "    #首先把动作计算出来\n",
    "    #[b, 3] -> [b, 1]\n",
    "    action = model_action(state)\n",
    "\n",
    "    #像value计算那里一样,拼合state和action综合计算\n",
    "    #[b, 3+1] -> [b, 4]\n",
    "    input = torch.cat([state, action], dim=1)\n",
    "\n",
    "    #使用value网络评估动作的价值,价值是越高越好\n",
    "    #因为这里是在计算loss,loss是越小越好,所以符号取反\n",
    "    #[b, 4] -> [b, 1] -> [1]\n",
    "    loss = -model_value(input).mean()\n",
    "\n",
    "    return loss\n",
    "\n",
    "\n",
    "get_loss_action(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_update(model, model_next):\n",
    "    for param, param_next in zip(model.parameters(), model_next.parameters()):\n",
    "        #以一个小的比例更新\n",
    "        value = param_next.data * 0.995 + param.data * 0.005\n",
    "        param_next.data.copy_(value)\n",
    "\n",
    "\n",
    "soft_update(torch.nn.Linear(4, 64), torch.nn.Linear(4, 64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "OHoSU6uI-xIt",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 400 -1481.9745946129065\n",
      "20 4400 -446.2614963239106\n",
      "40 8400 -165.38135583027767\n",
      "60 10000 -175.05355035907678\n",
      "80 10000 -176.24764820731824\n",
      "100 10000 -171.3202831257185\n",
      "120 10000 -137.3814099448686\n",
      "140 10000 -182.0192419057948\n",
      "160 10000 -93.50676630591069\n",
      "180 10000 -100.80603271275491\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    model_action.train()\n",
    "    model_value.train()\n",
    "    optimizer_action = torch.optim.Adam(model_action.parameters(), lr=5e-4)\n",
    "    optimizer_value = torch.optim.Adam(model_value.parameters(), lr=5e-3)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #训练N次\n",
    "    for epoch in range(200):\n",
    "        #更新N条数据\n",
    "        update_data()\n",
    "\n",
    "        #每次更新过数据后,学习N次\n",
    "        for i in range(200):\n",
    "            #采样一批数据\n",
    "            state, action, reward, next_state, over = get_sample()\n",
    "\n",
    "            #计算value和target\n",
    "            value = get_value(state, action)\n",
    "            target = get_target(next_state, reward, over)\n",
    "\n",
    "            #两者求差,计算loss,更新参数\n",
    "            loss_value = loss_fn(value, target)\n",
    "\n",
    "            optimizer_value.zero_grad()\n",
    "            loss_value.backward()\n",
    "            optimizer_value.step()\n",
    "\n",
    "            #使用value网络评估action网络的loss,更新参数\n",
    "            loss_action = get_loss_action(state)\n",
    "\n",
    "            optimizer_action.zero_grad()\n",
    "            loss_action.backward()\n",
    "            optimizer_action.step()\n",
    "\n",
    "            #以一个小的比例更新\n",
    "            soft_update(model_action, model_action_next)\n",
    "            soft_update(model_value, model_value_next)\n",
    "\n",
    "        if epoch % 20 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, len(datas), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAykklEQVR4nO3de3RTZb4//neSJmnaNGkLbUKhhSoVqNwLQvSs4xEqFdEZAWfUw0J0GC9YOCiMo4DoeDnAyPzE++V4lpeZNYoHFBQEsVO0KBSohQoUKEWBFtukpaVJr2kuz+8PJF/KRVu6mzxp36+1spbde+fZn/1Y8u7OfvazVUIIASIiIgmpQ10AERHRpTCkiIhIWgwpIiKSFkOKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFohC6nXXnsNAwYMQGRkJMaNG4fdu3eHqhQiIpJUSELqo48+woIFC/DUU09hz549GDFiBLKyslBVVRWKcoiISFKqUEwwO27cOIwdOxavvvoqAMDv9yM5ORnz5s3D448/HuxyiIhIUhHB3mFraysKCwuxaNGiwDK1Wo3MzEzk5+df9D1utxtutzvws9/vR21tLXr16gWVStXlNRMRkbKEEKivr0dSUhLU6kt/qRf0kDp16hR8Ph8sFkub5RaLBYcPH77oe5YvX46nn346GOUREVEQlZeXo1+/fpdcH/SQuhyLFi3CggULAj87nU6kpKSgvLwcJpMphJUREdHlcLlcSE5ORkxMzC9uF/SQ6t27NzQaDRwOR5vlDocDVqv1ou/R6/XQ6/UXLDeZTAwpIqIw9muXbII+uk+n0yEjIwO5ubmBZX6/H7m5ubDZbMEuh4iIJBaSr/sWLFiAWbNmYcyYMbjmmmvw4osvorGxEffee28oyiEiIkmFJKTuuOMOVFdX48knn4TdbsfIkSPxxRdfXDCYgoiIeraQ3CfVWS6XC2azGU6nk9ekiIjCUHs/xzl3HxERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtDocUtu2bcOtt96KpKQkqFQqrF+/vs16IQSefPJJ9OnTBwaDAZmZmSgtLW2zTW1tLWbMmAGTyYTY2FjMnj0bDQ0NnToQIiLqfjocUo2NjRgxYgRee+21i65//vnn8fLLL+PNN9/Erl27EB0djaysLLS0tAS2mTFjBoqLi5GTk4ONGzdi27ZtuP/++y//KIiIqHsSnQBArFu3LvCz3+8XVqtVrFy5MrCsrq5O6PV68eGHHwohhDh48KAAIAoKCgLbbN68WahUKvHTTz+1a79Op1MAEE6nszPlExFRiLT3c1zRa1LHjh2D3W5HZmZmYJnZbMa4ceOQn58PAMjPz0dsbCzGjBkT2CYzMxNqtRq7du26aLtutxsul6vNi4iIuj9FQ8putwMALBZLm+UWiyWwzm63IzExsc36iIgIxMfHB7Y53/Lly2E2mwOv5ORkJcsmIiJJhcXovkWLFsHpdAZe5eXloS6JiIiCQNGQslqtAACHw9FmucPhCKyzWq2oqqpqs97r9aK2tjawzfn0ej1MJlObFxERdX+KhlRqaiqsVityc3MDy1wuF3bt2gWbzQYAsNlsqKurQ2FhYWCbrVu3wu/3Y9y4cUqWQ0REYS6io29oaGjA0aNHAz8fO3YMRUVFiI+PR0pKCh5++GE899xzSEtLQ2pqKpYuXYqkpCTcdtttAIAhQ4bgpptuwn333Yc333wTHo8Hc+fOxZ133omkpCTFDoyIiLqBjg4b/OqrrwSAC16zZs0SQpwZhr506VJhsViEXq8XEydOFCUlJW3aqKmpEXfddZcwGo3CZDKJe++9V9TX1ys+dJGIiOTU3s9xlRBChDAjL4vL5YLZbIbT6eT1KSKiMNTez/GwGN1HREQ9E0OKiIikxZAiIiJpMaSIiEhaDCkiIpIWQ4qIiKTFkCIiImkxpIiISFoMKSIikhZDioiIpMWQIiIiaTGkiIhIWh1+VAcRyU34fPDU1sJz+jR8TU2AEFDr9YiIi4OuVy+odbpQl0jUbgwpom7EU1eHmtxcuIqK0HLyJDx1dYAQ0ERHI7JvXxjT09HrhhsQmZIClUoV6nKJfhVDiqgbEH4/Go8eRcU//oGGQ4cgWlvbrPfV16Px8GE0lpaibudO9L3nHsSOHQuVRhOiionahyFFFOaEEGg8fBgnXn8dLWVlv7yxzwd3RQXK33gDvoYGxF9/PdRabXAKJboMHDhBFOa8TidOvv/+rwfUOTynT6Pyww/RfOwYwvC5p9SDMKSIwpjw+1H54YdoPHy4w+9tra7GiVdfBfz+LqiMSBkMKaIw1vTjj6g/cAC4zLMht92O0/n5CldFpByGFFGYEj7fmVF85eWX3Ya/pQXOggIFqyJSFkOKKEz5Wlou62s+onDCkCIKU77GRjh37w51GURdiiFFFKZ8jY2hLoGoyzGkiMKUa+/eUJdA1OUYUkRhqvrzzzvdhkqrhWHAgM4XQ9RFGFJEYUj4fFDiFlyNwYC4665ToCWirsGQIgpDzeXlF8zPdzlUGg10CQkKVETUNRhSRGGovqjozGM4OslwxRUKVEPUdRhSRGFGCHHmTMrj6XRbcf/2bwpURNR1GFJEYcbX0ACvy6VIW1FXXqlIO0RdhSFFFGaaT5xA0w8/KNKWWqfjww9JagwpojAihDjzaPhTpzrdlikjAxExMQpURdR1GFJE4UQIeOvrFWkqevBgqA0GRdoi6ioMKaIw4mtuRk1uriJtaU0mqCP4cG6SG0OKKIwIjwdNR492viGNBioGFIUBhhRRGPG3tCjSjnHwYJhGj1akLaKuxJAiCiNNP/6oSDsaoxFas1mRtoi6EkOKKIzYP/lEkXY0UVH8uo/CAkOKKEwIr1eRZ0ipIyPRa+JEBSoi6noMKaIw0XrqlCJTIak0GkSlpipQEVHXY0gRhYnqL75Aa3V15xtSqaCJju58O0RBwJAiChP+1lZAdP4pUlEDBwKcConCBEOKKAz4WloUuR4FAL0zMxVphygYGFJEYaD11Cm0lJcr0lZkv36KtEMUDAwpojDgrqhQZqYJAFCrOfM5hQ2GFJHkhBCKPIUXODPzuTY2VpG2iIKBIUUkOeHxwLV3ryJtGQcP5uM5KKwwpIgkJzwe1OblKdKWWqeDSqNRpC2iYGBIEUlO+P2KtKNLTERUWpoibREFC0OKSHKNpaWK3B+l7dULhv79FaiIKHgYUkSSO/3NN4qElCYyktejKOwwpIgkJoRA8/HjirSl0ukUaYcomBhSRBLz1NTA73Z3uh2VVgvr7bcrUBFRcDGkiCTm2rMHrTU1nW5HpVZzpgkKSwwpIkkJIeC22+Fvbu50W+rISGj0egWqIgouhhSRpITHo9hME72zsgA1/7lT+OFvLZGkvPX1cNvtirQVdeWVfDwHhSWGFJGk3JWVqN+3T5G2tHFxirRDFGwMKSIJnZ1UVni9nW5Ll5AATVQUZz6nsMSQIpKR36/Y/VHG4cOh691bkbaIgo0hRSQh4fXCsW6dIm3pExKgNhgUaYso2BhSRBISPp8iN/EC4EMOKaxFhLoAop5ICNHmdXYZAKhUKrScPKnMpLK9e8N49dWdbocoVDp0JrV8+XKMHTsWMTExSExMxG233YaSkpI227S0tCA7Oxu9evWC0WjE9OnT4XA42mxTVlaGKVOmICoqComJiXj00UfhVeACMZGshBBoampCeXk59u/fj88++wwrVqzA/fffj5tuugmjRo3CFVdcAavVihtvvBHbn3kGwufr9H618fGIHjhQgSMgCo0OnUnl5eUhOzsbY8eOhdfrxeLFizFp0iQcPHgQ0dHRAIBHHnkEn3/+OdasWQOz2Yy5c+di2rRp2L59OwDA5/NhypQpsFqt2LFjByorK3H33XdDq9Vi2bJlyh8hUQj5fD4cPXoUe/bswXfffYc9e/agpKQE1dXV8F/iOVFutxt+j0eR/asjIqDh9SgKYyohLv87herqaiQmJiIvLw///u//DqfTiYSEBHzwwQe4/efJLA8fPowhQ4YgPz8f48ePx+bNm3HLLbegoqICFosFAPDmm2/iscceQ3V1NXTtmKnZ5XLBbDbD6XTCZDJdbvlEXcLv96O5uRlFRUVYvXo1vvnmG5w8eRJOpxN+vx9qtRoajQZ6vR4JCQlISkpCnz59YDabER0djf5WK8YePozIzs7Zp1aj76xZsE6dqsyBESmovZ/jnbom5XQ6AQDx8fEAgMLCQng8HmRmZga2GTx4MFJSUgIhlZ+fj2HDhgUCCgCysrIwZ84cFBcXY9SoURfsx+12w33ORWSXy9WZsom6hM/nQ1VVFXbs2IG3334b+fn5aGxsBAAYjUakpaVh4MCBGDlyJMaNG4f09HTEx8fDYDBApVIFBjfUFxWh/NAhdPZcSqVSwXSRf09E4eSyQ8rv9+Phhx/Gddddh6FDhwIA7HY7dDodYmNj22xrsVhg/3l6F7vd3iagzq4/u+5ili9fjqeffvpySyXqck1NTfjss8+wevVq5OXlweVyQaVSoV+/fpgwYQJsNhvGjBmDtLQ0REdH/+Jou5ajR+Gpre18USoV9FZr59shCqHLDqns7GwcOHAA3377rZL1XNSiRYuwYMGCwM8ulwvJycldvl+iXyKEgM/nw6FDh7Bq1Sps3LgRtbW10Gg0uPLKK3HHHXdg2rRpSE1NRUxMDDQaza+26fd64VNo6LlKq+XQcwp7lxVSc+fOxcaNG7Ft2zb0O+cZNVarFa2trairq2tzNuVwOGD9+S86q9WK3bt3t2nv7Og/6yX+6tPr9dDzMQMkESEETp8+jdWrV+OFF17A8ePHodFoMGjQIEybNg33338/+vbtGwiJ9oaFt64OjeeNmL1c1mnToIrgXSYU3jo0BF0Igblz52LdunXYunUrUlNT26zPyMiAVqtFbm5uYFlJSQnKyspgs9kAADabDfv370dVVVVgm5ycHJhMJqSnp3fmWIiCQgiBH3/8EU899RQWL16MY8eOwWAw4I9//CPeeecdLF26FMnJyVD/fBNtR85mPDU1aDhwQJE6Df378/EcFPY69GdWdnY2PvjgA3z66aeIiYkJXEMym80wGAwwm82YPXs2FixYgPj4eJhMJsybNw82mw3jx48HAEyaNAnp6emYOXMmnn/+edjtdjzxxBPIzs7m2RJJz+/34/vvv8fChQuRn58Pn8+HoUOHYvHixZg8eTJiYmIu+ys2IQTEJYalXw71zwMyiMJZh0LqjTfeAAD8x3/8R5vl7777Lu655x4AwKpVq6BWqzF9+nS43W5kZWXh9ddfD2yr0WiwceNGzJkzBzabDdHR0Zg1axaeeeaZzh0JURdrbW1FXl4e5s6di6NHj8JgMOCOO+7AkiVLMPDnG2Y7GwqNR48qUSq0vXtD8/O9i0ThrFP3SYUK75OiYPN6vdi8eTMWL16MQ4cOITY2Fg888ADmz5+PxMRERfYhfD4c/vOf0VRa2um2ek+ahH5//CM0kZEKVEakvKDcJ0XUE/h8PuTm5mLevHk4efIk+vTpg6VLl+I///M/AzOtKEH4fGgpL1ekLY3RCDW/PqdugCFF9At8Ph+2bduGhx56COXl5ejXrx9WrFiB3//+94GBEUpxOxyKXJNS6/XQJSbyehR1CwwpoksQQuDAgQN48sknceLECfTp0wdPPPEEpk2b1q57njrq1JdfKvIkXm1cHEzDhytQEVHocXwq0SU4HA4sXrwYO3fuhNlsxuLFizFz5sx2zS/ZUUKIM1/1KXEmFRUFfVKSAlURhR7PpIguorm5GatWrcKWLVug0WjwwAMPYPbs2V12m4S/uRn+1lbF2lPx/ijqJvibTHQev9+PTZs24e9//zuEEJg8eTLmzZvXpffxuR0OeH+esLmz4v/t3xRph0gGDCmicwghcPjwYaxatQrV1dUYOXIklixZcskpu5RSv3//mafxdpZKhdifZ3ch6g4YUkTn8Hg8+Pvf/46CggJERUVh3rx5GD16dJeOlBM+H/zNzYo8Lh4qFSLOewoBUThjSBH9TAiBgwcP4v3334fX68WNN96I22+/vUtG8p3L39KClspKRdqKGTECak4qS90IQ4roZ01NTVi5ciWqqqqQnJyMJUuWKHqz7qW0njqF2q+/VqQt04gRnPmcuhWGFBHOnEVt374deXl50Gq1uPPOOzF48OAuvyFWCAHh8yky9BzAmYcccmQfdSP8bSYCUF9fj/Xr16OqqgpXXHEFpk6dCoPBEJR9KzJgAgDUaj7okLodhhT1eGdH9G3ZsgUAMHnyZIwaNSpoH/bOggJF2okZMQJRV1yhSFtEsmBIUY/n9/vxr3/9C2VlZYiJicFdd93VJbNKXJQQcO7Zo0hTut69EcGnAlA3w5CiHq+pqQlr166F3+/Hddddh+FBnPfOU1ur2PUotU4HtVarSFtEsmBIUY9XUFCAgwcPQqvVYsaMGdAG8YPe+d13ikyHpNLpoLdYFKiISC4MKerRhBDYvHkzfD4frrzySowcOTKo+3ft3Qvh8XS6HW1sLGI5HRJ1Qwwp6tFOnz6N/Px8+P1+jBs3DlarNWgDJvytrYo8mgMAVFotdPHxirR1LiEE9uzZgxMnTijeNlF7MKSoRysuLkZFRQWioqKQkZEBo9EYtH03/fCDYsPPI/v27ZKZz1tbW/HKK6/g22+/hV+ha2dEHcGQoh5LCIHS0lLU1tYiPj4egwYN6vIpkM7VfPw43ApNhxR/ww2KtHMuIQR+/PFHbNmyBTt37kRTU5Pi+yD6NQwp6rE8Hg+OHDmChoYGJCYmIi0tLWj7Dsw0oZDoLqhdCIFPPvkENTU12L59O3766ScIJSbBJeoAhhT1WPX19Th27BiEEOjbt2+XP47jXH63Gy0//aRYe+ouuK+rsrISOTk58Hq9KCkpwffff8+QoqBjSFGP1djYiIqKCqjVaqSmpgbvBl4AvoYG1O3apUhbppEjFQ8pIQQKCwtx5MgRAEBLSws2b97M61IUdAwp6rEaGxtRWVkJtVqNq666Kmij+oQQ8Lvd8Jw6pUh7ptGjoVb4qcFNTU3Izc1FdXV1YNmXX34Jh8Oh6H6Ifg1DinqslpYW1NTUQKVSoX///kHd9y8NmPipsREby8vx4Y8/4l8VFWj8lfuotHFxUCk44EMIAbvdjtzc3DZnTqdOnUJOTo5i+yFqDz54hnosj8eDhoYGaLVaJCQkBHXf9k8+uWCZEALHGhrw1N69ON7QgBafDyatFkPj4vC3sWOhvdQQ8y44A9y1axdKS0vbLPP5fPjiiy9w5513IjIyUvF9El0Mz6SoxxJCwO/3Q6vVIjbIj1x3V1RcsOzHhgbct307DjmdaPb5IAA4PR5sr6rC/F27UNPScsF7TKNGwTh0qKK1eTyewNOJzyWEQFFREQ4cOMABFBQ0DCnq8dRqddDPDHpNmHDBsheLi+G8xFd7u0+dQs5Fgi0iNhZas1nR2oqLi1FYWHjRdRUVFdi9ezcHUFDQMKSox1Or1UG9iVelUqHXjTcq0pZaq1X0epTf78enn34a+BrU/HMAnu2jxsZG7NixA06nk2dTFBQMKerx/H4/fAreWBssmpgYxWeaKCsrQ25uLuLi4rBw4ULMnDkTGo0GQ4YMwdy5c9G3b1988803OKnU04SJfgVDino8v98Pt9sd6jIwJTkZ2ksMghhgNGL4eRPIqnU6GBQclXh2MlmPx4Nly5Zh0aJFMP38EMWYmBg8+OCD+Nvf/oaEhATsUugeL6Jfw9F91ON5vV40NDQEdZ9asxnxN9yA2q++CizLSkoCADz3/fdo9fngB6BRqRCr0+H/GzsW/c+b/Nb6u99BEx2tWE0tLS1wu93461//imuvvRYRERFoaGiAEAJRUVEwGAyYOnUqUlNTkZOTAyFE0O4to56LIUU9VkREBKKiouDxeFBVVRXUfasNBsTZbHB+9x189fUAzlyrykpKQr+oKGw8eRI1LS0YYDTijtRU9DrvZl19nz6Is9kUDQm9Xo+pU6dC//O+vF4vampqIIRAbGwsIiMjodVqMWbMGAwaNIgBRUHBkKIeKzIyEnFxcXA4HPhJwXn02kOlUsE0ahQSJk+G4+OPA5PNqlQqDI2Lw9C4uEu+NyIuDkkzZyLi56/ilHL+KMfW1lacOnWqTUgFald430SXwmtS1GMZDAZYLBb4/X788MMPQR+tptbrYZk6FfETJkAV0b6/FzVGI/rccQdir7lG0VF9F1NbW4va2lpoNBokJCQEzrCIgokhRT2W0WhEv379IITAkSNHLrh5NRg0UVHo94c/wDJ9OnSJiZfcTqXRwDBgAJLvuw8Jkyd3yazn5/vpp59w+vRp6PV69OvXD1qttsv3SXQ+ft1HPVZMTAxSUlIghEBFRQVqa2thsViCWoNKpUJEdDT63H47TMOH4/SOHWgoLobbboff7YbGaERkv34wjxkD85gxMKSkBOVakBACJ0+exOnTp2E0GpGcnMxrUBQSDCnqsSIjI5GWlgaDwQCHw4Fjx44FPaTOUuv1MA4diuirroKvuRnC64Xw+6HSaKDWaqGOioK6nV8JKsHr9aK0tBR1dXW48sorMXDgwKDtm+hcDCnq0QYNGoT4+HhUV1fj4MGDuOaaa6C+1ESuXUylUkGl1yv+2I3L0dDQgH379sHn8yElJQUpKSmhLol6KF6Toh5LpVJhyJAhSEhIQGNjIw4cOICWi0zi2tMIIVBbW4uioiKoVCqMHj0aUVFRoS6LeiiGFPVoffr0wejRoyGEwK5duwJDrnu6AwcO4IcffoBWq8XkyZN5PYpChiFFPZpGo8Ett9wCjUaD/fv3o6SkJNQlhZwQAp988gl8Ph8GDhyI4cOHh7ok6sEYUtTjXXvttejbty8aGxuxYcOGHn8mZbfb8dXP0zXddNNNiFZw6iWijmJIUY8XFxeH6dOnQ6VSYf369SgrKwt1SSHj9/uxZs0aVFdXw2KxYMqUKYgI4qhCovMxpKjH02g0uPnmm5GQkICqqir83//9X1g+ukMJlZWV2LBhAzweD2w2G4YMGcLrURRSDCnq8VQqFdLT03HdddfB6/Viw4YNOH78eI/72s/v92Pbtm3Yu3cvoqOjMWnSJPTu3TvUZVEPx5AiApCQkIDp06cjLi4Oe/fuxfr16+G5xKPcuyMhBKqqqvDOO++grq4OQ4YMwW9/+9ugPrGY6GIYUkQ4MwP4jTfeiNGjR6O5uRn//Oc/ceLEiR51NrV582Z8++23UKvVuOeee2C1WkNdEhFDiuisXr16Yc6cOYiMjMSBAwfwzjvvwO/3h7qsLieEQHl5OV588UW0trYiIyMD06ZNC3VZRAAYUkQBKpUKkyZNwtSpUyGEwNtvv42vvvqq2wdVfX09XnjhBRw+fBhxcXFYuHAhr0WRNBhSROcwGAx48MEHMXDgQNTV1eGFF17AiRMnQl1Wl/H5fNiyZQvWrl0LIQRuueUWTJgwIWTzFxKdj7+JROdQqVQYO3Ys7r33XkRFReHrr7/GW2+9hYaGhlCXpjghBI4dO4Zly5ahsrIS/fv3xyOPPIL4+PhQl0YUwJAiOo9Op8MDDzyAKVOmwO1245VXXsFHH30UkocidiWHw4E///nP2LdvH2JiYvCXv/wFw4cP531RJBWGFNF5VCoVzGYzlixZgquvvhotLS347//+b/zrX//qNten6urq8PLLL+PLL7+ETqfDzJkz8Zvf/IYBRdJhSBFdwpAhQ/Dss8+if//+KCsrw5IlS7Bt27awn42iqakJ//M//4M333wTra2tmDx5MhYsWACj0Rjq0oguwJAiugS1Wo2bb74ZS5cuRa9evVBUVIT58+djx44dYXtGVV9fj7fffhvPPvssnE4nbDYbli9fjgEDBvAsiqTEkCK6BJVKBa1WizvvvBMLFy4M3D/1pz/9KXBGFS43+woh4HK58NZbb2HZsmVoamrCiBEjsGzZMlx11VUMKJIWQ4roVxgMBsyZMwdLlixBXFwcCgsLMWfOHHz88cdwu92hLu9XCSFQXV2Np556Cs8++yxqa2sxduxYvPjiixg/fnyoyyP6RZyDn6gdYmJi8F//9V8wmUx47rnncOTIETz00EM4ceIEZs2ahYSEBCnPRrxeL/bt24e//vWv+PjjjwEANpsNq1atQkZGhpQ1E52LZ1JE7WQ0GjF79mysWLECffv2xenTp/Hcc89h4cKFOHjwIPx+vzRf/wkh4PV68dFHH+H+++/HunXroFarkZWVhZdeeokBRWFDJWT5V9UBLpcLZrMZTqcTJpMp1OVQD3L2w3/79u34y1/+gvz8fPj9fvTt2xdLly7Frbfeil69eoVs9nAhBFpbW1FSUoJXX30Vq1evRmNjI8xmM2bOnInHHnsMVquVM0pQyLX3c5whRXQZzs7WsHLlSqxduxa1tbUwGAyYMGECZs6ciUmTJsFsNge1Jq/Xix9//BFr167FP/7xD5SWlgaelTVnzhzcc889iIyMDGpNRJfCkCLqYkIInD59Gp9//jlefPFF7Nu3D0IIJCQk4Prrr8eMGTNwww03ICoqCiqVSvGv187+0/X7/SgrK8OHH36IDRs2YN++fWhpaYFer8fvfvc7ZGdnY+TIkdBqtfyKj6TR7s9x0QGvv/66GDZsmIiJiRExMTFi/PjxYtOmTYH1zc3N4qGHHhLx8fEiOjpaTJs2Tdjt9jZtnDhxQtx8883CYDCIhIQE8ac//Ul4PJ6OlCGcTqcAIJxOZ4feR6Q0v98vvF6vsNvt4oknnhCDBw8WkZGRQqPRiOjoaDFu3Djx0ksvie+++044HA7R2toq/H5/p/bp8/lEY2OjOH78uNi8ebPIzs4Wffv2FTqdTqjVahEbGysmTJgg1qxZIxobGzu9P6Ku0N7P8Q6dSW3YsAEajQZpaWkQQuD999/HypUrsXfvXlx99dWYM2cOPv/8c7z33nswm82YO3cu1Go1tm/fDuDMjMsjR46E1WrFypUrUVlZibvvvhv33Xcfli1bpnwCEwWRx+PBvn37sG7dOqxfvx6lpaXwer1Qq9Xo06cPMjIykJGRgSFDhuCqq65CcnIyoqOjERER8YtnOH6/H263G6dPn8axY8dw+PBhFBcXo6CgAPv27QtMfms0GjF27Fj8/ve/xy233II+ffrwzImkFbSv++Lj47Fy5UrcfvvtSEhIwAcffIDbb78dAHD48GEMGTIE+fn5GD9+PDZv3oxbbrkFFRUVsFgsAIA333wTjz32GKqrq6HT6RQ9OKJQaG5uRmlpKbZu3YoPPvgAxcXFaG1thd/vh1arhdlsRnx8PBISEpCUlISUlBQkJSUhLi4O0dHRUKlU8Hg8cDqdqK6uRkVFBcrKyuBwOFBbW4uamho0NjbC7/cjIiICRqMREydOxJ133okxY8agb9++fOw7Sa+9n+OXfZ+Uz+fDmjVr0NjYCJvNhsLCQng8HmRmZga2GTx4MFJSUgIhlZ+fj2HDhgUCCgCysrIwZ84cFBcXY9SoURfdl9vtbnPTpMvlutyyibqcwWDAsGHDcPXVV+O+++7Dzp07sWnTJhQUFKC8vBynTp1CTU0Njhw5ErhWdfaM59wzHyFEmxdwZqqm2NhYpKWl4corr8SkSZMwadIkpKamBs7IePZE3UmHQ2r//v2w2WxoaWmB0WjEunXrkJ6ejqKiIuh0OsTGxrbZ3mKxwG63AwDsdnubgDq7/uy6S1m+fDmefvrpjpZKFDIqlQoajQbR0dGYOHEiJkyYgKqqKhw6dAilpaU4evQojh8/jsrKSjgcDjidTjQ1NcHtdkMIAbVaDYPBAKPRiPj4ePTp0wf9+vVDamoqBg0ahLS0NFx11VUwGAwMJerWOhxSgwYNQlFREZxOJ9auXYtZs2YhLy+vK2oLWLRoERYsWBD42eVyITk5uUv3SaQklUoFi8UCi8WC66+/Hm63G42NjWhubkZzczNaW1vh9XoDNwSfDTmtVgu9Xg+DwYCoqChER0dDo9EwmKjH6HBI6XQ6DBw4EACQkZGBgoICvPTSS7jjjjvQ2tqKurq6NmdTDocDVqsVAGC1WrF79+427TkcjsC6S9Hr9dDr9R0tlUhKKpUKkZGRvGeJqB06fdv52ZFHGRkZ0Gq1yM3NDawrKSlBWVkZbDYbgDNzhu3fvx9VVVWBbXJycmAymZCent7ZUoiIqJvp0JnUokWLMHnyZKSkpKC+vh4ffPABvv76a2zZsgVmsxmzZ8/GggULEB8fD5PJhHnz5sFmswVmWp40aRLS09Mxc+ZMPP/887Db7XjiiSeQnZ3NMyUiIrpAh0KqqqoKd999NyorK2E2mzF8+HBs2bIFN954IwBg1apVUKvVmD59OtxuN7KysvD6668H3q/RaLBx40bMmTMHNpsN0dHRmDVrFp555hllj4qIiLoFTotERERB197PcU6FTERE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkrU6F1IoVK6BSqfDwww8HlrW0tCA7Oxu9evWC0WjE9OnT4XA42ryvrKwMU6ZMQVRUFBITE/Hoo4/C6/V2phQiIuqGLjukCgoK8NZbb2H48OFtlj/yyCPYsGED1qxZg7y8PFRUVGDatGmB9T6fD1OmTEFrayt27NiB999/H++99x6efPLJyz8KIiLqnsRlqK+vF2lpaSInJ0dcf/31Yv78+UIIIerq6oRWqxVr1qwJbHvo0CEBQOTn5wshhNi0aZNQq9XCbrcHtnnjjTeEyWQSbre7Xft3Op0CgHA6nZdTPhERhVh7P8cv60wqOzsbU6ZMQWZmZpvlhYWF8Hg8bZYPHjwYKSkpyM/PBwDk5+dj2LBhsFgsgW2ysrLgcrlQXFx80f253W64XK42LyIi6v4iOvqG1atXY8+ePSgoKLhgnd1uh06nQ2xsbJvlFosFdrs9sM25AXV2/dl1F7N8+XI8/fTTHS2ViIjCXIfOpMrLyzF//nz885//RGRkZFfVdIFFixbB6XQGXuXl5UHbNxERhU6HQqqwsBBVVVUYPXo0IiIiEBERgby8PLz88suIiIiAxWJBa2sr6urq2rzP4XDAarUCAKxW6wWj/c7+fHab8+n1ephMpjYvIiLq/joUUhMnTsT+/ftRVFQUeI0ZMwYzZswI/LdWq0Vubm7gPSUlJSgrK4PNZgMA2Gw27N+/H1VVVYFtcnJyYDKZkJ6ertBhERFRd9Cha1IxMTEYOnRom2XR0dHo1atXYPns2bOxYMECxMfHw2QyYd68ebDZbBg/fjwAYNKkSUhPT8fMmTPx/PPPw26344knnkB2djb0er1Ch0VERN1BhwdO/JpVq1ZBrVZj+vTpcLvdyMrKwuuvvx5Yr9FosHHjRsyZMwc2mw3R0dGYNWsWnnnmGaVLISKiMKcSQohQF9FRLpcLZrMZTqeT16eIiMJQez/HOXcfERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUmLIUVERNJiSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUkrItQFXA4hBADA5XKFuBIiIrocZz+/z36eX0pYhlRNTQ0AIDk5OcSVEBFRZ9TX18NsNl9yfViGVHx8PACgrKzsFw+up3O5XEhOTkZ5eTlMJlOoy5EW+6l92E/tw35qHyEE6uvrkZSU9IvbhWVIqdVnLqWZzWb+ErSDyWRiP7UD+6l92E/tw376de05yeDACSIikhZDioiIpBWWIaXX6/HUU09Br9eHuhSpsZ/ah/3UPuyn9mE/KUslfm38HxERUYiE5ZkUERH1DAwpIiKSFkOKiIikxZAiIiJphWVIvfbaaxgwYAAiIyMxbtw47N69O9QlBdW2bdtw6623IikpCSqVCuvXr2+zXgiBJ598En369IHBYEBmZiZKS0vbbFNbW4sZM2bAZDIhNjYWs2fPRkNDQxCPomstX74cY8eORUxMDBITE3HbbbehpKSkzTYtLS3Izs5Gr169YDQaMX36dDgcjjbblJWVYcqUKYiKikJiYiIeffRReL3eYB5Kl3rjjTcwfPjwwI2nNpsNmzdvDqxnH13cihUroFKp8PDDDweWsa+6iAgzq1evFjqdTrzzzjuiuLhY3HfffSI2NlY4HI5QlxY0mzZtEkuWLBGffPKJACDWrVvXZv2KFSuE2WwW69evF99//734zW9+I1JTU0Vzc3Ngm5tuukmMGDFC7Ny5U3zzzTdi4MCB4q677grykXSdrKws8e6774oDBw6IoqIicfPNN4uUlBTR0NAQ2ObBBx8UycnJIjc3V3z33Xdi/Pjx4tprrw2s93q9YujQoSIzM1Ps3btXbNq0SfTu3VssWrQoFIfUJT777DPx+eefiyNHjoiSkhKxePFiodVqxYEDB4QQ7KOL2b17txgwYIAYPny4mD9/fmA5+6prhF1IXXPNNSI7Ozvws8/nE0lJSWL58uUhrCp0zg8pv98vrFarWLlyZWBZXV2d0Ov14sMPPxRCCHHw4EEBQBQUFAS22bx5s1CpVOKnn34KWu3BVFVVJQCIvLw8IcSZPtFqtWLNmjWBbQ4dOiQAiPz8fCHEmT8G1Gq1sNvtgW3eeOMNYTKZhNvtDu4BBFFcXJz43//9X/bRRdTX14u0tDSRk5Mjrr/++kBIsa+6Tlh93dfa2orCwkJkZmYGlqnVamRmZiI/Pz+Elcnj2LFjsNvtbfrIbDZj3LhxgT7Kz89HbGwsxowZE9gmMzMTarUau3btCnrNweB0OgH8v8mJCwsL4fF42vTT4MGDkZKS0qafhg0bBovFEtgmKysLLpcLxcXFQaw+OHw+H1avXo3GxkbYbDb20UVkZ2djypQpbfoE4O9TVwqrCWZPnToFn8/X5n8yAFgsFhw+fDhEVcnFbrcDwEX76Ow6u92OxMTENusjIiIQHx8f2KY78fv9ePjhh3Hddddh6NChAM70gU6nQ2xsbJttz++ni/Xj2XXdxf79+2Gz2dDS0gKj0Yh169YhPT0dRUVF7KNzrF69Gnv27EFBQcEF6/j71HXCKqSILkd2djYOHDiAb7/9NtSlSGnQoEEoKiqC0+nE2rVrMWvWLOTl5YW6LKmUl5dj/vz5yMnJQWRkZKjL6VHC6uu+3r17Q6PRXDBixuFwwGq1hqgquZzth1/qI6vViqqqqjbrvV4vamtru10/zp07Fxs3bsRXX32Ffv36BZZbrVa0trairq6uzfbn99PF+vHsuu5Cp9Nh4MCByMjIwPLlyzFixAi89NJL7KNzFBYWoqqqCqNHj0ZERAQiIiKQl5eHl19+GREREbBYLOyrLhJWIaXT6ZCRkYHc3NzAMr/fj9zcXNhsthBWJo/U1FRYrdY2feRyubBr165AH9lsNtTV1aGwsDCwzdatW+H3+zFu3Lig19wVhBCYO3cu1q1bh61btyI1NbXN+oyMDGi12jb9VFJSgrKysjb9tH///jaBnpOTA5PJhPT09OAcSAj4/X643W720TkmTpyI/fv3o6ioKPAaM2YMZsyYEfhv9lUXCfXIjY5avXq10Ov14r333hMHDx4U999/v4iNjW0zYqa7q6+vF3v37hV79+4VAMQLL7wg9u7dK06cOCGEODMEPTY2Vnz66adi37594re//e1Fh6CPGjVK7Nq1S3z77bciLS2tWw1BnzNnjjCbzeLrr78WlZWVgVdTU1NgmwcffFCkpKSIrVu3iu+++07YbDZhs9kC688OGZ40aZIoKioSX3zxhUhISOhWQ4Yff/xxkZeXJ44dOyb27dsnHn/8caFSqcSXX34phGAf/ZJzR/cJwb7qKmEXUkII8corr4iUlBSh0+nENddcI3bu3BnqkoLqq6++EgAueM2aNUsIcWYY+tKlS4XFYhF6vV5MnDhRlJSUtGmjpqZG3HXXXcJoNAqTySTuvfdeUV9fH4Kj6RoX6x8A4t133w1s09zcLB566CERFxcnoqKixNSpU0VlZWWbdo4fPy4mT54sDAaD6N27t1i4cKHweDxBPpqu84c//EH0799f6HQ6kZCQICZOnBgIKCHYR7/k/JBiX3UNPqqDiIikFVbXpIiIqGdhSBERkbQYUkREJC2GFBERSYshRURE0mJIERGRtBhSREQkLYYUERFJiyFFRETSYkgREZG0GFJERCQthhQREUnr/we4JtkkSmsWigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-129.54026289274628"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第7章-DQN算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
